{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取数据\n",
    "train = pd.read_csv('FE_pima-indians-diabetes.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.866045       -0.031990   \n",
       "1  -0.844885                     -1.205066       -0.528319   \n",
       "2   1.233880                      2.016662       -0.693761   \n",
       "3  -0.844885                     -1.073567       -0.528319   \n",
       "4  -1.141852                      0.504422       -2.679076   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.670643      -0.181541  0.166619   \n",
       "1                    -0.012301      -0.181541 -0.852200   \n",
       "2                    -0.012301      -0.181541 -1.332500   \n",
       "3                    -0.695245      -0.540642 -0.633881   \n",
       "4                     0.670643       0.316566  1.549303   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']   \n",
    "X_train = train.drop([ \"Target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "#查看一个学习器是否支持稀疏数据，可以看fit函数是否支持: X: {array-like, sparse matrix}.\n",
    "#可自行用timeit比较稠密数据和稀疏数据的训练时间\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 分离X和y\n",
    "y_train = train['Target']\n",
    "X_train = train.drop(['Target'],axis=1)\n",
    "\n",
    "feat_names = X_train.columns\n",
    "\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is: [0.48805562 0.53007069 0.45629158 0.42247552 0.48388972]\n",
      "cv logloss is: 0.47615662604739944\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr,X_train,y_train,cv = 5,scoring = 'neg_log_loss')\n",
    "print('logloss of each fold is:',-loss)\n",
    "print('cv logloss is:',-loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 正则化的Logistic Regression + GridSearchCV及参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 14 candidates, totalling 70 fits\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................ C=0.001, penalty=l1, score=-0.693, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................ C=0.001, penalty=l1, score=-0.693, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................ C=0.001, penalty=l1, score=-0.693, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................ C=0.001, penalty=l1, score=-0.693, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................ C=0.001, penalty=l1, score=-0.693, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................ C=0.001, penalty=l2, score=-0.625, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................ C=0.001, penalty=l2, score=-0.634, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................ C=0.001, penalty=l2, score=-0.626, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................ C=0.001, penalty=l2, score=-0.621, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................ C=0.001, penalty=l2, score=-0.633, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l1, score=-0.636, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l1, score=-0.645, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l1, score=-0.633, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l1, score=-0.630, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l1, score=-0.636, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l2, score=-0.512, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l2, score=-0.543, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l2, score=-0.507, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l2, score=-0.487, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ................. C=0.01, penalty=l2, score=-0.525, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l1, score=-0.489, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l1, score=-0.525, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l1, score=-0.458, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l1, score=-0.433, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l1, score=-0.487, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l2, score=-0.484, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l2, score=-0.525, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l2, score=-0.460, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l2, score=-0.429, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .................. C=0.1, penalty=l2, score=-0.485, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] .................... C=1, penalty=l1, score=-0.488, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] .................... C=1, penalty=l1, score=-0.529, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] .................... C=1, penalty=l1, score=-0.456, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] .................... C=1, penalty=l1, score=-0.422, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] .................... C=1, penalty=l1, score=-0.485, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .................... C=1, penalty=l2, score=-0.488, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] .................... C=1, penalty=l2, score=-0.530, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] .................... C=1, penalty=l2, score=-0.456, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] .................... C=1, penalty=l2, score=-0.423, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] .................... C=1, penalty=l2, score=-0.484, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ................... C=10, penalty=l1, score=-0.489, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ................... C=10, penalty=l1, score=-0.531, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ................... C=10, penalty=l1, score=-0.456, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ................... C=10, penalty=l1, score=-0.422, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ................... C=10, penalty=l1, score=-0.484, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ................... C=10, penalty=l2, score=-0.489, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ................... C=10, penalty=l2, score=-0.531, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ................... C=10, penalty=l2, score=-0.456, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ................... C=10, penalty=l2, score=-0.422, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ................... C=10, penalty=l2, score=-0.484, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] .................. C=100, penalty=l1, score=-0.489, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] .................. C=100, penalty=l1, score=-0.531, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] .................. C=100, penalty=l1, score=-0.456, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] .................. C=100, penalty=l1, score=-0.422, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] .................. C=100, penalty=l1, score=-0.484, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .................. C=100, penalty=l2, score=-0.489, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .................. C=100, penalty=l2, score=-0.531, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .................. C=100, penalty=l2, score=-0.456, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .................. C=100, penalty=l2, score=-0.422, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .................. C=100, penalty=l2, score=-0.484, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ................. C=1000, penalty=l1, score=-0.489, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ................. C=1000, penalty=l1, score=-0.531, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ................. C=1000, penalty=l1, score=-0.456, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ................. C=1000, penalty=l1, score=-0.422, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ................. C=1000, penalty=l1, score=-0.484, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ................. C=1000, penalty=l2, score=-0.489, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ................. C=1000, penalty=l2, score=-0.531, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ................. C=1000, penalty=l2, score=-0.456, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ................. C=1000, penalty=l2, score=-0.422, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ................. C=1000, penalty=l2, score=-0.484, total=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  70 out of  70 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score=nan,\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='auto',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='liblinear',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='deprecated', n_jobs=None,\n",
       "             param_grid={'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
       "                         'penalty': ['l1', 'l2']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.001, 0.01,0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver ='liblinear')\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss',verbose=5)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.47596795430530375\n"
     ]
    }
   ],
   "source": [
    "print(-grid.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3yUZbr/8c+VTAqB0DuBgAgqxSBECKKu2MC2riirKNjWZbHses6en7ueXVePenTr2eNZxV7XhliwImLHQgtIRxFRIRRBkCqQdv3+mEGzcUImZXgyk+/79ZpXZp6573m+D2WuPO2+zd0RERGpLCXoACIi0jCpQIiISFQqECIiEpUKhIiIRKUCISIiUYWCDlCf2rZt6927dw86hohIwpg3b97X7t4u2ntJVSC6d+9OYWFh0DFERBKGmX1Z1XtxPcRkZiPN7BMzW2lm10Z5/xozWxB5LDGzMjNrHUtfERGJr7gVCDNLBSYCpwB9gDFm1qdiG3f/q7sPcPcBwH8C77r7llj6iohIfMVzD2IwsNLdV7l7MTAJOHM/7ccAT9ayr4iI1LN4noPoAqyp8LoIGBKtoZllASOBq2rRdzwwHqBbt251SywiSaOkpISioiL27NkTdJQGITMzk5ycHNLS0mLuE88CYVGWVTXw0xnAB+6+paZ93f1e4F6A/Px8DSwlIgAUFRWRnZ1N9+7dMYv2ldJ4uDubN2+mqKiIHj16xNwvnoeYioCuFV7nAOuqaHse3x9eqmlfEZEf2LNnD23atGn0xQHAzGjTpk2N96biWSDmAr3MrIeZpRMuAi9WbmRmLYAfAS/UtK+IyP6oOHyvNn8WcSsQ7l5K+JzCa8ByYLK7LzWzCWY2oULTs4Dp7r6rur7xynr7m5+yZO22eH28iCSIc++Zybn3zAw6RoMR1/sg3H2qu/d2957ufktk2d3ufneFNg+7+3mx9I2Hb3YV88Sc1Zx7z0zeXbEpXqsRkUaoWbNm3z0fOXIkLVu25PTTT4/a9sorr2TAgAH06dOHJk2aMGDAAAYMGMAzzzxTo3XOnz+fadOm1Sn3Po1+LKZWTdOZcsUwurbO4tKH5zK5cE31nUREauiaa67h0UcfrfL9iRMnsmDBAqZOnUrPnj1ZsGABCxYs4JxzzqnRelQg6lnHFpk8PWEoQw9qw2+eWcRtb6xAM+2JSH064YQTyM7OrlXfTz/9lBEjRjBo0CCOPfZYVqxYAcCkSZPo168feXl5DB8+nN27d3PTTTfx+OOP12rvo7KkGoupLrIz03jw4iO59rlF3PbGp6zbuptbzupPWqpqqEiiu/GlpSxbt73adsvWh9vEch6iT+fm3HBG3zpni8X48eO5//776dmzJx988AFXXXUV06dP58Ybb+Sdd96hQ4cObN26lSZNmnD99dezZMkSbrvttjqvVwWigvRQCv8zOo+clk34x1sr2bB9L3deMJBmGfpjEpFgbN26lVmzZnH22Wd/t6y0tBSAYcOGceGFFzJ69GhGjRpV7+vWN18lZsavTz6ETi2bcN3zSzj3npk8dPGRtG+eGXQ0EamlWH/T37fn8NQvhsYzTo24O23btmXBggU/eO++++5j9uzZvPzyy+Tl5bFo0aJ6XbeOn1RhzOBu3H9hPp9/vYuz7vyQlRt3BB1JRBqhVq1a0alTJ6ZMmQJAeXk5CxcuBGDVqlUUFBRw880306pVK9auXUt2djY7dtTP95UKxH4MP7Q9T40fyt7Sckbd+SGzV20OOpKIJKhjjjmG0aNH8+abb5KTk8Nrr70Wc99JkyZx9913k5eXR9++fXn55ZcB+Pd//3f69+9P//79OfHEE+nXrx/HH388Cxcu5IgjjqjzSWpLpqt18vPzPR4TBq3Z8i0XPTSHoi27+fu5eZx+eOd6X4eI1K/ly5dz2GGH1ahPQzzEVJ+i/ZmY2Tx3z4/WXucgYtC1dRbPXX4Ulz1SyFVPfMT6rXu47Jgeuo1fJMkka2GoLR1iilHLrHQeu2wIp/bvyC1Tl3PjS8soK0+evS8RkcpUIGogMy2VO8YM5GdH9+DhD7/gisfnsaekLOhYIiJxoQJRQykpxh9O78P1p/dh+rKvOP++WWzZVRx0LBGReqcCUUuXHt2DO88fyJJ12zn7rg/5cvOu6juJiCQQFYg6OKV/J564bAjffFvMqDs/ZMGarUFHEpG6eOi08EMAFYg6y+/emmcvP4qsjFTOu3cmbyz7KuhIItJA7Bvue8GCBQwdOpS+ffty+OGH89RTT/2gbUMc7luXudaDnu2a8dzlw/jZI3MZ/2ghN53Zj7EFuUHHEpEGIisri3/+85/06tWLdevWMWjQIEaMGEHLli2/azNx4kQAvvjiC04//fSoQ2vEYv78+SxZsoSRI0fWObf2IOpJu+wMJo0v4LhD2nPd80v487SPKddlsCIC9O7dm169egHQuXNn2rdvz6ZNsU9QpuG+k0BWeoh7xw3iDy8s5a53PmP91t385Zw80kOqwyKBevVa2LC4+nYbIoPdxXIeomN/OOVPNY4yZ84ciouL6dmzZ8x9NNx3kgilpnDrWf3IadWEv772CRt37OXucYNonpkWdDQRCdj69esZN24cjzzyCCkpsf3iqOG+k4yZceXwg+nUIpPfPLOI0XfN5KFLjqRzyyZBRxNpnGL9TX/fnsMlr9R7hO3bt3Paaafx3//93xQUFMTcT8N9J6lRA3N45NLBrNu6m1F3fsjy9dXPaCUiyae4uJizzjrru9/2a0LDfSexYQe3ZfKE8ABgo++eyQcrvw44kYgcaJMnT2bGjBk8/PDD312+WpOrlDTcdz2I13Df9WH9tt1c/OBcPtu0k7+cczijBuYEHUkkqdVmuO94HmJqCGo63Lf2IA6QTi2a8PTlQxncozW/nryQO976lGQqziJJ4ZJXkrY41IYKxAHUPDONhy8ZzFlHdOFv01fwuymLKS0rDzqWiEhUuorpAEsPpfD3n+bRuWUmE9/+jA3b9nDH+QNpmqG/CpH65u6a2CuiNkcstAcRADPjmhGHcstZ/Xh3xSbOu3cWm3bsDTqWSFLJzMxk8+bNOpRLuDhs3ryZzMzMGvXTr60BumBILh2bZ3LVEx8x6q4PePiSwfRs1yzoWCJJIScnh6KiohoNaZHMMjMzycmp2cUxuoqpAVi4Zis/e2QupeXO/Rfmk9+9ddCRRKSR0FVMDVxe15Y8d/kwWmWlc/79s3l18fqgI4mIxLdAmNlIM/vEzFaa2bVVtDnOzBaY2VIze7fC8i/MbHHkvcTbLaihbm2yePbyo+jXuTlXPDGfB97/POhIItLIxa1AmFkqMBE4BegDjDGzPpXatATuBH7s7n2ByvegD3f3AVXt/iSb1k3TeeLnBYzo05GbX17GTS8t05DhIhKYeO5BDAZWuvsqdy8GJgFnVmpzPvCcu68GcPeNccyTEDLTUpl4wUAuGdadBz/4nKuenM+ekrKgY4lIIxTPAtEFWFPhdVFkWUW9gVZm9o6ZzTOzCyu858D0yPLxVa3EzMabWaGZFSbL1QqpKcYNZ/TlutMOY+riDYx7YDZbvy0OOpaINDLxLBDR7k6pfLwkBAwCTgNGAH8ws96R94a5+0DCh6iuNLNjo63E3e9193x3z2/Xrl09RW8YLjvmIO44/wgWrtnGqLs+ZM2Wb4OOJCKNSDwLRBHQtcLrHGBdlDbT3H2Xu38NzADyANx9XeTnRmAK4UNWjc7ph3fmscuGsHlnMWfd+SGLi7YFHUlEGol4Foi5QC8z62Fm6cB5wIuV2rwAHGNmITPLAoYAy82sqZllA5hZU+BkYEkcszZog3u05tnLh5IRSuHce2fy9seN/lSNiBwAcSsQ7l4KXAW8BiwHJrv7UjObYGYTIm2WA9OARcAc4H53XwJ0AN43s4WR5a+4+7R4ZU0EB7fPZsqVR3FQu6Zc9s9CnpyzOuhIIpLkdCd1gtm1t5QrHp/Puys28cvjD+bXJ/XWYGQiUmu6kzqJNM0Icf9F+Zyb35Xb31rJfzy9kOJSDRkuIvVPg/UloLTUFP50dn+6tGrC319fwcbte7lr7ECyM9M4956ZADz1i6EBpxSRRKc9iARlZvzqhF78bXQes1ZtZvTdM9mwbU/QsUQkiahAJLhzBuXw0CVHUvTNbs668wO+LS4NOpKIJAkViCRwTK92PPWLAsrKnWXrd2hoDhGpFyoQSaJv5xY8e/lRlLvrUJOI1AsViCTStXUWrbPS2bRzL7v26lCTiNSNCkSS6dA8g3KH5xesDTqKiCQ4FQiAh04LP5JAs4wQWempPDrzS03WLiJ1ogKRZCZPOIrrTuvDxxt2MH/1N0HHEZEEpgLhDt9uhpLkGUr7zAGdaZYR4rFZGq9JRGpPBWLPNti8ArasCheLJNA0I8TZA7vwyqL1bN65N+g4IpKgVCCatIQW3WDPVljxWtBp6s3YglyKy8qZXFgUdBQRSVAqEADZnSDUBF77HZQmx9SevTpkM6RHa56Y8yVl5cmxZyQiB5YKBIClQOsesOUzmHNP0GnqzbihuazZspsZK5Jjrm4RObBUIAAueQWumAm9ToZ3/wI7k+ML9eQ+HWnbLIPHZn0ZdBQRSUAqEBWNuDV8NdNbNwedpF6kh1IYM7grb32ykTVbkucqLRE5MFQgKmrbCwb/Aub/E9YvCjpNvRgzuBsGmqJURGpMBaKyH/0GslrDtGuT4rLXzi2bcMJhHXhq7hr2lmqUVxGJnQpEZU1awvHXwZcfwLIXgk5TL8YV5LJ5VzHTlmwIOoqIJBAViGgGXgQd+sH0P0DJ7qDT1NnRB7clt02WTlaLSI2oQESTkgoj/wjbVsPMO4JOU2cpKcbYIbnM/eIbPt6wPeg4IpIgVCCq0uNYOOwMeO/vsH1d0Gnq7JxBOaSHUrQXISIxU4HYn5NuhvIyeOPGoJPUWaum6ZxxeGemzF/Ljj0lQccRkQSgArE/rXvA0Cth0SRYMzfoNHU2bmguu4rLeP4jTSYkItVTgajOMb+GZh3Dl72Wlwedpk7yclrQr0tzHpu1WpMJiUi1VCCqk5ENJ94Aawth8eSg09SJmTGuIJdPvtrB3C80mZCI7J8KRCwOPw86D4Q3/gv27gw6TZ2ckdeZ7MyQTlaLSLVUIGKRkgKn/Bl2rIf3/zfoNHWSlR7inEE5vLpkPZt2aDIhEamaCkSsug6G/qPhw9vhm8T+7XtsQS4lZc7kwjVBRxGRBiyuBcLMRprZJ2a20syuraLNcWa2wMyWmtm7Nel7wJ14Y/gmutevDzpJnfRs14yjerbhidmrNZmQiFQpbgXCzFKBicApQB9gjJn1qdSmJXAn8GN37wuMjrVvIFp0gWH/Bsuehy/eDzpNnYwryGXt1t28/fHGoKOISAMVzz2IwcBKd1/l7sXAJODMSm3OB55z99UA7r6xBn2DcdQvoUXXyGWviTs66ol9OtA+O4PHZif24TIRiZ94FoguQMWD3EWRZRX1BlqZ2TtmNs/MLqxBXwDMbLyZFZpZ4aZNB2AmuPQsOOlG2LAYPno0/uuLk7TUFMYM7sa7KzaxerMmExKRH4pngbAoyyof8A4Bg4DTgBHAH8ysd4x9wwvd73X3fHfPb9euXV3yxq7vKOg2FN68GfZsOzDrjIMxg7uRYsbjc7QXISI/FM8CUQR0rfA6B6g86l0RMM3dd7n718AMIC/GvsExg5F/gm83h+ewTlAdW2Ry0mEdmDx3DXtKEvdwmYjERzwLxFygl5n1MLN04DzgxUptXgCOMbOQmWUBQ4DlMfYNVucBcMRYmH0PfL0y6DS1Nm5oLt98W8LUxeuDjiIiDUzcCoS7lwJXAa8R/tKf7O5LzWyCmU2ItFkOTAMWAXOA+919SVV945W11k64HkKZMP33QSeptaN6tuGgtk11Z7WI/IAl06Bt+fn5XlhYeGBX+sH/he+LGPssHHzigV13PXng/c+5+eVlvPKro+nbuUXQcUTkADKzee6eH+093UldV0MmQKseMO13UJaY8yycMzCHzLQUHpu1OugoItKAqEDUVSgDRtwKX38ChQ8GnaZWWmSl8eO8zjz/0Vq2azIhEYlQgagPh5wCBx0Hb98K324JOk2tjCvozu6SMqbM12RCIhKmAlEfzGDEH2HvjnCRSED9c1qQl9OCR2d9qcmERARQgag/HfpA/qVQ+AB8tSzoNLUytiCXlRt3MmtVYu4FiUj9qnGBMLMUM2sejzAJb/jvIKN5eJymBPwt/Iy8zrRokqbxmUQEiLFAmNkTZtbczJoCy4BPzOya+EZLQFmtw0Xi83fhk6lBp6mxzLRURg/K4bUlG9i4fU/QcUQkYLHuQfRx9+3AT4CpQDdgXNxSJbL8S6HdofDa76E08WZsu6Agl9Jy56m5mkxIpLGLtUCkmVka4QLxgruXUMXgeY1ealr4stdvPodZdwWdpsZ6tG3KMb3a8sSc1ZSWlQcdR0QCFGuBuAf4AmgKzDCzXGB7vEIlvINPgN6nwIy/wY6vgk5TY2MLclm/bQ9vaTIhkUYtpgLh7v9w9y7ufqqHfQkMj3O2xDbiFijdA2/dFHSSGjvh0PZ0apHJoxqfSaRRi/Uk9dWRk9RmZg+Y2Xzg+DhnS2xtekLBBPjocVj3UdBpaiQUmUzovU+/5vOvdwUdR0QCEushpksjJ6lPBtoBlwB/iluqZHHsNZDVBl5NvMtezzuyK6EU4wld8irSaMVaIPbN8HYq8JC7LyT6rG9SUWaL8JDga2bB0ueCTlMj7ZtnMqJvRyYXFmkyIZFGKtYCMc/MphMuEK+ZWTagS1xiccRY6Ngfpl8PxYk19/PYgly27S7h5UWaTEikMYq1QPwMuBY40t2/BdIJH2aS6qSkwsg/w/Yi+PD2oNPUSMFBrTm4fTOdrBZppGK9iqmc8LzQ15nZ34Cj3H1RXJMlk+7DoM9P4P3/hW1FQaeJmZkxdkg3Fq7ZyuKibUHHEZEDLNarmP4EXE14mI1lwK/M7I/xDJZ0TroJcHjjv4JOUiOjBuXQJC1VU5KKNEKxHmI6FTjJ3R909weBkcBp8YuVhFrlwlG/hMVPw+rZQaeJWfPMNH5yRGdeWLiWbd9qMiGRxqQmo7m2rPBcExfXxtH/DtmdYdpvoTxxzvFfMCSXPSXlPDs/cQ6PiUjdxVog/gh8ZGYPm9kjwDwgMWfGCVJ6Uzjxv8I3zi18Mug0MevXpQVHdGvJY7M1mZBIYxLrSeongQLguchjqLtPimewpNV/NOQcCW/eGJ6BLkGMK8hl1aZdzPxsc9BRROQA2W+BMLOB+x5AJ6AIWAN0jiyTmkpJCV/2uvMreO9/gk4Ts1P7d6JVVpoueRVpRELVvL+/bzBH4zHVTs4gyBsDMyfCwIugdY+gE1UrMy2Vn+Z35f73P2fDtj10bJEZdCQRibP97kG4+/D9PFQc6uKEGyAlDaZfF3SSmJ0/pBtl5c6kuauDjiIiB0Cs90GMivI4wczaxztg0mreCY75NXz8Mqx6N+g0Mclt05Qf9W7Hk3NWU6LJhESSXk2G2rgfuCDyuA/4NfCBmWnq0doaehW07AbT/hPKSoNOE5NxBbl8tX0vby5PvImQRKRmYi0Q5cBh7n62u58N9AH2AkOA38YrXNJLy4STboaNS2H+I0GnicnwQ9vTpWUTnawWaQRiLRDd3b3ir4wbgd7uvgXQ7bV10edMyD0a3vpv2P1N0GmqlZpinD+kGx+s3Mxnm3YGHUdE4ijWAvGemb1sZheZ2UXAi4Tnpm4KbI1fvEbADEb+MVwc3v1L0Gli8tP8rqSlGo/P0slqkWQWa4G4EngIGAAcATwCXOnuu9y9yrmpzWykmX1iZivN7Noo7x9nZtvMbEHkcX2F974ws8WR5YU126wE0+lwGHQRzLkXNq0IOk212mVnMLJfJ56Zt4bdxZpMSCRZxXontQPvA28BbwAzvJoxF8wsFZgInEL4nMUYM+sTpel77j4g8rip0nvDI8vzY8mZ0IZfB2lZ8Nrvgk4Sk7FDurF9TykvLVwXdBQRiZNYL3P9KTAHOAf4KTDbzM6ppttgYKW7r3L3YmAScGZdwia1Zu3gR7+Fla/DiulBp6nW4B6t6d1BkwmJJLNYDzH9nvBsche5+4WEv/z/UE2fLoSH5dinKLKssqFmttDMXjWzvhWWOzDdzOaZ2fiqVmJm482s0MwKN23aFNvWNFSDx0Obg8N7EWUN+9y/mTGuIJfFa7excI1OQ4kko1gLRIq7b6zwenMMfS3KssqHpeYDue6eB9wOPF/hvWHuPpDwIaorzezYaCtx93vdPd/d89u1a1dNpAYulA4jboXNn8Kc+4JOU62fHNGFrPRU7UWIJKlYC8Q0M3vNzC42s4uBV4Cp1fQpArpWeJ0D/MsBa3ff7u47I8+nAmlm1jbyel3k50ZgCuG9luTX62ToeQK88yfY9XXQafYrOzONs47owksL17H12+Kg44hIPYv1JPU1wL3A4UAecK+7V3eD3Fygl5n1MLN04DzCl8d+x8w6mplFng+O5NlsZk3NLDuyvClwMrAk9s1KYPsuey3eCW/fEnSaao0tyGVvaTnPzNNkQiLJprrRXL/j7s8Cz9agfamZXQW8BqQCD7r7UjObEHn/bsInvS83s1JgN3Ceu7uZdQCmRGpHCHjC3afFuu6E1+4QGPzz8GWv+T+Djv2CTlSlwzo1Jz+3FY/N+pJLh/UgJSXakUURSUS2v6tVzWwHPzxvAOHzC+7uzeMVrDby8/O9sDBJbpnY/Q38YyB06AsXvRTes2igXliwlqsnLeDRnw3mmF4Jfh5IpJExs3lV3UpQ3XDf2e7ePMoju6EVh6TTpBUM/x188R4sfynoNPs1sl9H2jRN59GZOlktkkxiPUktQRh0CbTvE54zomRP0GmqlBFK5adHduWN5V+xftvuoOOISD1RgWjIUkPhE9Zbv4RZE4NOs1/nD+6GA0/O1vhMIslCBaKhO+g4OPR0mPE/sH190Gmq1LV1FsMPac+Tc9doMiGRJKECkQhOvhnKS+DNykNVNSxjC7qxacdepi/VZEIiyUAFIhG0PggKroCFT8DaeUGnqdKPercnp1UTHp31RdBRRKQeqEAkimP/HzRtD69eC/sfSDcwqSnGBUNymbVqCys37gg6jojUkQpEosjIhhNvgKI5sPiZqts9dFr4EZCf5ueQnprCY5pMSCThqUAkkrzzodMAeP16KN4VdJqo2jTL4NT+HXl2XhG79pYGHUdE6kAFIpGkpMApf4Yd6+CD/ws6TZXGFuSyY28pL2oyIZGEpgKRaLoVQL+zwwVia8M8jDMotxWHdszm0ZlfUs3EgyLSgKlAJKITbwQMXr8h6CRRmRnjhuaybP12PtJkQiIJSwUiEbXsCsOuhqXPwZcfBp0mqp8M6EKzjBCPaXwmkYSlApGohl0NzbvAq7+F8rKg0/xA04wQowZ24eXF69myS5MJiSQiFYhElZ4FJ90EGxbBgseDThPV2IJcikvLebpwTfWNRaTBUYFIZP3Ohq4F4SE49mwPOs0P9O6QzeAerXl89mrKy3WyWiTRqEAksn3Tk+7aBDP+GnSaqMYW5LJ6y7fM+HRT0FFEpIZUIBJdl4EwYCzMugs2fxZ0mh8Y2bcjbZul89gsnawWSTQqEMnghOshlBGeWKiBSQ+lcN6R3Xjr440UffNt0HFEpAZUIJJBdofwYH6fTA3PZd3AjBnSDYAn5zTMG/tEJDoViGRRcAW06g7ffA7esCbs6dKyCccf2oGn5q6huLRhZRORqqlAJItQBoy4FUq+hfUfwWdvB53oX4wt6MbXO4uZtnRD0FFEJEYqEMnk0NOg3WHh+SIe/QlMugC+aRgnh4/t1Y5urbN0Z7VIAlGBSDZZbaDzQDj+D/DZWzBxMLx9KxQHe4I4JcUYW9CNOV9s4ZMNmkwoUS299WiW3np00DHqLFm2A+K7LSoQychSwietr5oLh5wK7/45XCiWvRDobHSjB3UlPZTS6C55TaYvI2lcVCCSWYscGP0QXPwKZDSHyRfCP38MG5cHEqdV03ROP7wTUz5ay05NJiTS4KlANAbdj4ZfzIBT/wbrF8Fdw8JzW+8+8ENxjy3IZefeUp7/aO1+2517z0zOvWfmAUolItGoQDQWqSEY/HP45XwYeCHMvhtuHwTz/wnlB+7S0yO6tqRv5+Y8Nmv/kwldv/kart98zQHLJSI/pALR2DRtA2fcBr94F9ocDC/+Eu4/HooKD8jqzYyxBbl8vGEH875seDf1icj3VCAaq055cOk0OOte2L4e7j8Bnr8CdnwV91WfOaAz2RkhHm1kJ6tFEk1cC4SZjTSzT8xspZldG+X948xsm5ktiDyuj7Wv1AMzyDsXflkIw/4NFk0OH3b68A4oK4nbarPSQ5w9KIdXF2/g651747YeEambuBUIM0sFJgKnAH2AMWbWJ0rT99x9QORxUw37Sn3IyIaTboQrZkG3Apj+e7jrqPB9FHEytqAbxWXlTNZkQiINVjz3IAYDK919lbsXA5OAMw9AX6mttgfDBU/DmKfCexCPnhW5G/uLel/Vwe2zGXpQG56YvZoyTSYk0iDFs0B0ASr+elgUWVbZUDNbaGavmlnfGvbFzMabWaGZFW7apElp6swMDhkZ3ps44frwXsQdg+GtW+r9buyxBbkUfbObd1dsrNfPFZH6Ec8CYVGWVf5VcT6Q6+55wO3A8zXoG17ofq+757t7frt27WodVipJy4Rj/gOuKoTDzoAZfwnfjb30+Xq7G/vkvh1ol53BoxqfSaRBimeBKAK6VnidA6yr2MDdt7v7zsjzqUCambWNpa8cIC26wDkPwMVTIbMFPH1R+G7sr5bV+aPTUlMYM7gb76zYxJotmkxIpKGJZ4GYC/Qysx5mlg6cB7xYsYGZdTQzizwfHMmzOZa+UoVLXgk/6lv3YTD+3e/vxr77aHj1t3W+G3vM4K6kmPH4bE0mJNLQxK1AuHspcBXwGrAcmOzuS81sgplNiDQ7B1hiZguBfwDneVjUvvHKKjHadzf2rz6CQRfB7Hvg9oEw75Fa343dqUUTTjysPZML17C3tKyeA4tIXcT1Pgh3n+ruvd29p7vfEll2t7vfHXl+h7v3db4ubawAAAwUSURBVPc8dy9w9w/311caiKzWcPr/hu/GbtsbXvpV+G7sNXNr9XFjC3LZsquYVxdrMiGRhkR3UkvtdcqDS16FUffDjg3wwIkw5fIa3409rGdberRtqjurRRoYFQipGzM4fHR47olh/waLn47cjX07lBbH9BEpKcYFQ7ox78tvWLZue5wDi0isVCCkfuy7G/vK2ZA7FKZfB3cPg5VvxtT9nEE5ZIRSeGy29iJEGgoVCKlfbXqG78Y+fzKUl8Jjo+DJ82HL5/vt1jIrnR/ndeb5j9ayY0/8xoESkdipQEh89B4RuRv7Blj1DkwcUu3d2GMLcvm2uIwp1UwmJCIHhgqExE8oA475dfj8RJ8fh+/GvuNIWPJc1Lux87q25PCcFpHJhALIKyL/QgVC4q9FFzj7/vAVT01awTOXwCNnRL0be+yQXFZ8tZOlZV2jfJCIHEgqEHLg5B4VvnfitP+Br5aE78ae+hvY/f3McmfkdaZ5ZohXigcGGFREQAVCDrSUVDjysvDc2IMuhrn3hS+LnfcwlJfRJD2V0fld+bD0ELaUNw06rUijpgIhwchqDaf/PTy+U9ve8NLVcN/xsGYOFwzpRimpTC/JCzqlSKMWCjqANHKdDg+fm1j8DLz+B3jgJA7KG8Pw1G48vXcoH93+Hk3SUslMS6VJWipN0lO/fx153iQtlcwKz5ukp/xL+6y0EJnpKd+9H0rV70UisVCBkODtuxv7kFPgvb/BzIncm1bG9NTB7C7rQ3EJ7C03ikudveWwtxT2lsOuMueb8hTKMcpJodwjP9m3zCgjBY/8LI88t5RUQqFU0lJDhNJCpIVChEIh0kMh0iOv00Mh0tNTSQ+lkZ4WIj09jYxQiIy0tPDz9BAZaSEy09PJSAuRkZ5OZkaIJmlpZKankZkewlJSwVK/v2KrrCTy3Pfzs7zSMqK2cS/Hy51yL8fdKXfH3fHyfa/Lw8/Ly3En8trD/Qi329eH8vLvPif8GZF2vu8nlJeXgTubSzNx4NPC13/49/hdXI+2uEKDaB0r9fHK77Of93+4zqhrr9BkQ0lWeK0fvFRFv8SxoSSLFKBvtS1rzqr+g008+fn5XlhYGHQMqavNn7Hjjh+R7TuCTiKSEL725rS9sXbzu5vZPHfPj/ae9iCk4WnTk9Vp3cGdvr95M/JbdTl4WfhneXmUZWUVllV4fLd830+PsmxfO4/6me7llJSWUlJSSnFJCSWlpRSXllFaWkJJaRklpaWUlpZSGllWWlZOaWkZZWWllJaVsnPtchwjs0MvwtOfGJiFn1d8TUr4te17HW7jZpilRJqmRNqnfPdZ+z7nX3+mYJG24cXff/b3n2VgqZgZKT/o+/36sRQsJdxv44wHMKD9cT//7q/rX6d/tGgLv3vpkWeRaWAqvV9Fp+9eVupjlT6z8r8jq9Swwmesf/3/AOh00tWVeyWcDa/fhhkcH4fPVoGQhsssPPVp0DGA9MijNtdVLb31aAD6Xvn3ekwVjKWzbgOg73GjAk5SR+/cCkDfo04NOEg9iGxLPOhsnYiIRKUCISIiUalAiIhIVCoQIiISlQqEiIhEpQIhIiJRqUCIiEhUKhAiIhKVCoSIiESlAiEiIlGpQIiISFQqECIiEpUKhIiIRKUCISIiUalAiIhIVHEtEGY20sw+MbOVZnbtftodaWZlZnZOhWVfmNliM1tgZpomTkTkAIvbhEFmlgpMBE4CioC5Zvaiuy+L0u7PwGtRPma4u38dr4wiIlK1eO5BDAZWuvsqdy8GJgFnRmn3S+BZYGMcs4iISA3Fs0B0ASrOol0UWfYdM+sCnAXcHaW/A9PNbJ6Zja9qJWY23swKzaxw06ZN9RBbREQgvgXiB3OIE/7Sr+g24LfuXhal7TB3HwicAlxpZsdGW4m73+vu+e6e365du7olFhGR78TtHAThPYauFV7nAOsqtckHJpkZQFvgVDMrdffn3X0dgLtvNLMphA9ZzYhjXmlA+nZqEXQEkUYvnnsQc4FeZtbDzNKB84AXKzZw9x7u3t3duwPPAFe4+/Nm1tTMsgHMrClwMrAkjllFRKSSuO1BuHupmV1F+OqkVOBBd19qZhMi70c777BPB2BKZM8iBDzh7tPilVUaoEteCTqBSKMXz0NMuPtUYGqlZVELg7tfXOH5KiAvntlERGT/dCe1iIhEpQIhIiJRmXvlK08TV35+vhcWalQOEZFYmdk8d8+P9p72IEREJCoVCBERiUoFQkREolKBEBGRqFQgREQkKhUIERGJSgVCRESiUoEQEZGoVCBERCSqpLqT2sw2AV/WsntbIFnmv06WbUmW7QBtS0OULNsBdduWXHePOttaUhWIujCzwqpuN080ybItybIdoG1piJJlOyB+26JDTCIiEpUKhIiIRKUC8b17gw5Qj5JlW5JlO0Db0hAly3ZAnLZF5yBERCQq7UGIiEhUKhAiIhKVCkQFZnazmS0yswVmNt3MOgedqTbM7K9m9nFkW6aYWcugM9WWmY02s6VmVm5mCXdJopmNNLNPzGylmV0bdJ66MLMHzWyjmS0JOktdmFlXM3vbzJZH/m1dHXSm2jKzTDObY2YLI9tyY71+vs5BfM/Mmrv79sjzXwF93H1CwLFqzMxOBt5y91Iz+zOAu/824Fi1YmaHAeXAPcD/c/eEmVPWzFKBFcBJQBEwFxjj7ssCDVZLZnYssBP4p7v3CzpPbZlZJ6CTu883s2xgHvCTRPx7MTMDmrr7TjNLA94Hrnb3WfXx+dqDqGBfcYhoCiRk9XT36e5eGnk5C8gJMk9duPtyd/8k6By1NBhY6e6r3L0YmAScGXCmWnP3GcCWoHPUlbuvd/f5kec7gOVAl2BT1Y6H7Yy8TIs86u17SwWiEjO7xczWABcA1wedpx5cCrwadIhGqguwpsLrIhL0iyhZmVl34AhgdrBJas/MUs1sAbAReN3d621bGl2BMLM3zGxJlMeZAO7+e3fvCjwOXBVs2qpVtx2RNr8HSglvS4MVy7YkKIuyLCH3SpORmTUDngX+rdLRg4Ti7mXuPoDwkYLBZlZvh/9C9fVBicLdT4yx6RPAK8ANcYxTa9Vth5ldBJwOnOAN/ERTDf5OEk0R0LXC6xxgXUBZpILI8fpngcfd/bmg89QHd99qZu8AI4F6uZCg0e1B7I+Z9arw8sfAx0FlqQszGwn8Fvixu38bdJ5GbC7Qy8x6mFk6cB7wYsCZGr3Iid0HgOXu/veg89SFmbXbd5WimTUBTqQev7d0FVMFZvYscAjhq2a+BCa4+9pgU9Wcma0EMoDNkUWzEvFqLAAzOwu4HWgHbAUWuPuIYFPFzsxOBW4DUoEH3f2WgCPVmpk9CRxHeGjpr4Ab3P2BQEPVgpkdDbwHLCb8fx3gd+4+NbhUtWNmhwOPEP73lQJMdveb6u3zVSBERCQaHWISEZGoVCBERCQqFQgREYlKBUJERKJSgRARkahUIERqwMx2Vt9qv/2fMbODIs+bmdk9ZvZZZCTOGWY2xMzSI88b3Y2s0rCoQIgcIGbWF0h191WRRfcTHvyul7v3BS4G2kYG9nsTODeQoCIRKhAitWBhf42MGbXYzM6NLE8xszsjewQvm9lUMzsn0u0C4IVIu57AEOA6dy8HiIz6+kqk7fOR9iKB0S6sSO2MAgYAeYTvLJ5rZjOAYUB3oD/QnvBQ0g9G+gwDnow870v4rvCyKj5/CXBkXJKLxEh7ECK1czTwZGQkza+Adwl/oR8NPO3u5e6+AXi7Qp9OwKZYPjxSOIojE9qIBEIFQqR2og3lvb/lALuBzMjzpUCeme3v/2AGsKcW2UTqhQqESO3MAM6NTNbSDjgWmEN4ysezI+ciOhAe3G6f5cDBAO7+GVAI3BgZXRQz67VvDgwzawNscveSA7VBIpWpQIjUzhRgEbAQeAv4TeSQ0rOE54FYQnge7dnAtkifV/jXgnEZ0BFYaWaLgfv4fr6I4UDCjS4qyUWjuYrUMzNrFplEvg3hvYph7r4hMl7/25HXVZ2c3vcZzwH/mcDzcUsS0FVMIvXv5cgkLunAzZE9C9x9t5ndQHhe6tVVdY5MLvS8ioMETXsQIiISlc5BiIhIVCoQIiISlQqEiIhEpQIhIiJRqUCIiEhU/x9DIJeq+QIVqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "#train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "#train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "#train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "#train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    #plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(grid.best_estimator_, open(\"diabetes_L1_org.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 正确率做指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 14 candidates, totalling 70 fits\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................. C=0.001, penalty=l1, score=0.649, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................. C=0.001, penalty=l1, score=0.649, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................. C=0.001, penalty=l1, score=0.649, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................. C=0.001, penalty=l1, score=0.654, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ................. C=0.001, penalty=l1, score=0.654, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................. C=0.001, penalty=l2, score=0.734, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................. C=0.001, penalty=l2, score=0.714, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................. C=0.001, penalty=l2, score=0.753, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................. C=0.001, penalty=l2, score=0.784, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ................. C=0.001, penalty=l2, score=0.752, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l1, score=0.682, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l1, score=0.682, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l1, score=0.734, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l1, score=0.725, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l1, score=0.712, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l2, score=0.753, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l2, score=0.708, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l2, score=0.773, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l2, score=0.804, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .................. C=0.01, penalty=l2, score=0.771, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l1, score=0.760, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l1, score=0.740, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l1, score=0.773, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l1, score=0.784, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l1, score=0.758, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l2, score=0.766, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l2, score=0.747, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l2, score=0.792, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l2, score=0.797, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ................... C=0.1, penalty=l2, score=0.771, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ..................... C=1, penalty=l1, score=0.760, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ..................... C=1, penalty=l1, score=0.747, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ..................... C=1, penalty=l1, score=0.786, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ..................... C=1, penalty=l1, score=0.797, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ..................... C=1, penalty=l1, score=0.771, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ..................... C=1, penalty=l2, score=0.760, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ..................... C=1, penalty=l2, score=0.740, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ..................... C=1, penalty=l2, score=0.786, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ..................... C=1, penalty=l2, score=0.797, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ..................... C=1, penalty=l2, score=0.771, total=   0.0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] .................... C=10, penalty=l1, score=0.760, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] .................... C=10, penalty=l1, score=0.740, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] .................... C=10, penalty=l1, score=0.786, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] .................... C=10, penalty=l1, score=0.797, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] .................... C=10, penalty=l1, score=0.771, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] .................... C=10, penalty=l2, score=0.760, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] .................... C=10, penalty=l2, score=0.740, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] .................... C=10, penalty=l2, score=0.786, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] .................... C=10, penalty=l2, score=0.797, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] .................... C=10, penalty=l2, score=0.771, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ................... C=100, penalty=l1, score=0.760, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ................... C=100, penalty=l1, score=0.740, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ................... C=100, penalty=l1, score=0.786, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ................... C=100, penalty=l1, score=0.797, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ................... C=100, penalty=l1, score=0.771, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ................... C=100, penalty=l2, score=0.760, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ................... C=100, penalty=l2, score=0.740, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ................... C=100, penalty=l2, score=0.786, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ................... C=100, penalty=l2, score=0.797, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ................... C=100, penalty=l2, score=0.771, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .................. C=1000, penalty=l1, score=0.760, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .................. C=1000, penalty=l1, score=0.740, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .................. C=1000, penalty=l1, score=0.786, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .................. C=1000, penalty=l1, score=0.797, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .................. C=1000, penalty=l1, score=0.771, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .................. C=1000, penalty=l2, score=0.760, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .................. C=1000, penalty=l2, score=0.740, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .................. C=1000, penalty=l2, score=0.786, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .................. C=1000, penalty=l2, score=0.797, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .................. C=1000, penalty=l2, score=0.771, total=   0.0s\n",
      "0.7747644512350395\n",
      "{'C': 0.1, 'penalty': 'l2'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  70 out of  70 | elapsed:    0.2s finished\n"
     ]
    }
   ],
   "source": [
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.001,0.01,0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "#缺省scoring为正确率\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5,verbose=5)\n",
    "grid.fit(X_train,y_train)\n",
    "lr_best=grid.best_estimator_\n",
    "print(grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这里为什么L1正则下评价分数全是nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l2\n",
      "0.1\n"
     ]
    }
   ],
   "source": [
    "print(grid.best_params_['penalty'])\n",
    "print(grid.best_params_['C'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd5xU9fX/8deRKqDBAhYUSzTIUpayWJAqoUXB2FGUREWCNAVRROQHBjEkMQYLIkj8WiBqJCioqIgFUWxEkCqIiBRRUBAposCe3x8zwLDsDrPL3rkzs+/n4zEPd2bu3Dl7H3LPftr5mLsjIiJSkEPCDkBERFKbEoWIiMSlRCEiInEpUYiISFxKFCIiEpcShYiIxBVYojCzx8xsnZktKOB9M7MHzGyZmc0zswZBxSIiIkUXZIvicaBdnPfbA6dHH92A0QHGIiIiRRRYonD3d4ANcQ65EHjSIz4AKpvZcUHFIyIiRVM6xO+uBqyKeb46+travAeaWTcirQ4qVqzY8IwzzkhKgCKS2nasW0aZnZvDDiOlrd3ifLPFyXW+c/cqRTlHmInC8nkt33oi7j4WGAuQk5Pjs2fPDjIuEUmSNaMuoNr6mQd5lkrFEktea6o0pVrPlwI5dzK4O2bGlClTmDZtGqNGjfqqqOcKM1GsBk6MeX4C8HVIsYhIHMVzQ99ftWI4R1A39OKILQwbN26kf//+nHrqqQwaNIiOHTvSsWNHRo0aVeRzhpkopgC9zOwZ4Cxgk7vv1+0kIsmVX1II8qZ5sDf6dL2hB+H555+nR48erF+/njvvvLPYzhtYojCzp4EWwNFmthoYApQBcPdHgKnA74BlwDbg2qBiEZHEWwUF3Xj1l3vq+vbbb+nduzfPPfcc9erV4+WXX6ZBg+JbcRBYonD3Kw/wvgM9g/p+Edk3ORTmhpxfUtANPXWtWrWKl19+meHDh3PrrbdSpkyZYj1/mF1PIlJMCmot5L25J9oqUFJIfV999RUvvvgivXr1Iicnh5UrV3LUUUcF8l1KFCIpqjADyPFu7LHJQQkg/eXm5jJ69Ghuv/12AC655BKOO+64wJIEKFGIpJSidhVBwa0FJYfMsWTJErp27cq7775L27ZtGTNmDMcdF/w6ZSUKkRSStwVRmAFkJYTMtm3bNpo0acKuXbt4/PHH6dKlC2b5LUcrfkoUIilgv26moZsA3fwFli5dyumnn06FChV46qmnqFevHscee2xSY1CZcZEUEJsk1lRpGmIkkiq2b9/OoEGDyMrKYsKECQC0a9cu6UkC1KIQKXYHtYp56Ca1IoT33nuP66+/niVLlnDttddy/vnnhxqPEoVIAYp6wy/qjX5NlaZKEsKwYcMYMmQI1atX57XXXqNNmzZhh6REIVKQg6ltVJRVzEoSJdvuIn716tWjd+/eDB8+nEqVgil4WFhKFCIHEh1YLgzd9CVRGzZsoG/fvpx22mkMHjyYDh060KFDh7DD2ocShQjBVUcViWfixIn07NmTDRs2MHjw4LDDKZAShQgFdzNp3ECCsHbtWnr16sWkSZNo2LAh06ZNIzs7O+ywCqREISXemlEX7E0GebqZlCQkCF9//TWvvfYaf/3rX+nXrx+lS6f2rTi1oxNJgt2tCbUeJEgrVqzgxRdfpHfv3jRs2JBVq1ZxxBFHhB1WQrTgTkqsNaMugKG/2vM8nbe9lNS1a9cuHnjgAWrXrs2gQYP45ptvANImSYAShZRAuxOEVkNL0BYvXkyzZs246aabaNq0KQsWLAhlZfXBUteTlAgFVWXdvd5BXU5S3LZt20azZs3Izc3lySef5Oqrr05aEb/ipkQhGW13gihoAx8lCClun332GTVq1KBChQpMmDCB7OxsjjnmmLDDOijqepKMtl/30tBNkXpKGo+QYvbTTz8xYMAAatWqtaeIX5s2bdI+SYBaFJKh8ivbrdaDBOWdd96ha9eufP7553Tt2pULLrgg7JCKlVoUkpE0UC3Jctddd9G8eXN27tzJ9OnTefTRR6lcuXLYYRUrtSgkLSVcckMtCQnI7iJ+OTk59O3bl2HDhlGxYsWwwwqEWhSSlhJJEmpJSBC+++47rrnmGoYNGwbA+eefz3333ZexSQLUopA0FK/kRiy1JKQ4uTvPPfccvXr1YuPGjQwZMiTskJJGiUJSWn5dTLsTgEpuSLJ8/fXX9OjRg8mTJ5OTk8P06dOpW7du2GEljRKFpKSC1j/seb8IGwOJFNU333zDm2++yd///nduvvnmlC/iV9xK1m8raSPvrKW8SUEtCQna8uXLmTJlCjfffDMNGjRg5cqVGTebKVFKFJISCpzFpFlLkmS7i/gNGjSIMmXK0KlTJ4499tgSmyRAs54kReSXJDRrSZJt4cKFnHvuufTr14/zzjuPhQsXpmURv+KmFoWklphZTGpJSDJt27aN5s2bY2b8+9//plOnTmlbxK+4KVFIqLRXtYRt0aJF1KxZkwoVKvDMM8+QnZ1NlSpVwg4rpajrSUKhPSEkbNu2bePWW2+lTp06jB8/HoDf/va3ShL5UItCki5vK0IlvyXZ3n77bW644QaWLVvGn/70Jzp27Bh2SClNiUKSIr+Ng5QgJAxDhgzhz3/+M7/+9a958803admyZdghpTwlCgnUgTYOEkmW3UX8zjzzTG655Rb+/Oc/U6FChbDDSgvm7sGd3KwdcD9QChjn7iPyvP8rYDxQnUjSutfd/y/eOXNycnz27NkBRSzFqaAuJpFkWr9+PTfddBM1atQoUfWZ8jKz/7l7TlE+G1iLwsxKAaOA1sBq4GMzm+Lui2IO6wkscvcOZlYFWGJmE9z9l6DikuDkTQzqYpIwuTtPP/00ffr04ccff+Suu+4KO6S0FeSspzOBZe6+PHrjfwa4MM8xDhxmkcnKlYANwM4AY5KAFDTNVa0ICcPq1avp2LEjnTt35rTTTmPOnDkMHDgw7LDSVpBjFNWAVTHPVwNn5TnmIWAK8DVwGHCFu+fmPZGZdQO6AVSvXj2QYOXg7E4SeRODWhEShvXr1/POO+9w33330adPH0qVKhV2SGktyBZFfksa8w6ItAXmAscD9YCHzOzw/T7kPtbdc9w9R3OcU8vu9RC7qfUgYVm2bBn//Oc/Aahfvz6rVq2ib9++ShLFIMgWxWrgxJjnJxBpOcS6FhjhkRH1ZWb2JXAG8FGAcclBym+qK2h/CAnHzp07GTlyJIMHD6ZcuXJcddVVHHPMMRx++H5/c0oRBdmi+Bg43cxOMbOyQCci3UyxVgKtAMzsGKAGsDzAmKQY5B2LWFOlaaTKq1oTkmTz58+ncePG3HrrrbRp04aFCxdyzDHHhB1WxgmsReHuO82sF/Aakemxj7n7QjPrHn3/EWAY8LiZzSfSVTXA3b8LKiY5OPsNWEcL+KkVIWHYtm0bLVu25JBDDuGZZ57h8ssvVxG/gAS64M7dpwJT87z2SMzPXwNtgoxBDo66mSTVLFiwgFq1alGhQgWeffZZsrOzOfroo8MOK6NpZbbso6C1EHve15oICcnWrVsZPHgwI0eO5IknnuCaa66hVatWYYdVIihRyD4OtBZCCULC8MYbb3DDDTfw5Zdf0qNHDy68MO+SLAmSEoUABY8/gJKDhGvw4MHcfffdnH766cyYMYNmzZqFHVKJo/0oBED7QkjKyc2NrL1t3Lgxt912G59++qmSREgCLQoYBBUFLB4F7iwX05IQCcO6devo06cPNWrUUH2mYnQwRQHVoihB9qyizrOz3J731ZKQELk748ePp2bNmjz//PMqAZ5CNEZRQuTXglBdJkkVq1atonv37kydOpVzzjmHcePGkZWVFXZYEqVEUULkV7RPiUFSxffff897773H/fffT8+ePVWfKcUoUWS4/dZFqMyGpIilS5cyZcoU+vfvT7169Vi1ahWHHXZY2GFJPjRGkaF2j0doNpOkmp07d/LXv/6VunXrMnz4cL799lsAJYkUphZFhslvj2qtppZU8emnn3LdddfxySefcNFFFzFq1CgV8UsDShQZJr89qpUgJBVs27aNVq1aUbp0aSZOnMgll1wSdkiSICWKDJHfymolCEkF8+bNo06dOlSoUIHnnnuO7OxsjjzyyLDDkkLQGEWG0FiEpJotW7Zw0003Ua9ePZ566ikAWrZsqSSRhtSiyDRqSUgKeP311+nWrRsrVqygV69eXHTRRWGHJAchoRaFmZU1s9OCDkZE0t+gQYNo06YN5cqVY+bMmTz44IOa0ZTmDpgozOx8YD7wevR5PTN7PujAJHFrRl0Qdggie4r4NWnShIEDBzJ37lyaNGkSclRSHBJpUfwZOAv4AcDd5wJqXaSI2EFsjU1IGL755hsuvfRShg4dCkD79u255557KF++fLiBSbFJJFHscPcf8ryWXiVnM1TeJKFV15JM7s7jjz9OVlYWL730EocffnjYIUlAEhnMXmxmlwOHmNkpwE3AB8GGJYlQkpCwfPXVV3Tr1o1p06bRpEkTxo0bR40aNcIOSwKSSIuiF9AQyAUmAduJJAtJEUoSkmw//PADH3/8MQ899BAzZsxQkshwibQo2rr7AGDA7hfM7GIiSUNCUOCmQyIBWrJkCVOmTOHWW28lOzublStXUqlSpbDDkiRIpEVxZz6vDSruQOTAVOhPwrBjxw7+8pe/kJ2dzYgRI1i3bh2AkkQJUmCLwszaAu2AamZ2X8xbhxPphpIkUx0nSbY5c+Zw/fXXM2fOHC699FIeeughqlatGnZYkmTxup7WAQuIjEksjHl9M3B7kEHJAWj1tSTBtm3baN26NWXKlOG///0vF198cdghSUgKTBTuPgeYY2YT3H17EmOSfKwZdYGSgyTFnDlzqFevHhUqVGDixIlkZ2dzxBFHhB2WhCiRMYpqZvaMmc0zs6W7H4FHJsD+4xIak5CgbN68mV69etGgQYM9RfxatGihJCEJzXp6HLgbuBdoD1yLxigCF28DIpHi9uqrr/KnP/2JVatWcdNNN6mbSfaRSIuigru/BuDuX7j7nUDLYMOS/WY2Dd2kJCGBGDhwIO3bt6dixYq89957jBw5UjOaZB+JtCh+NjMDvjCz7sAaQNMekkUD1xKQXbt2UapUKVq0aEHp0qW58847KVeuXNhhSQpKJFH0BSoBfYDhwK+A64IMKtNpwZyEae3atfTs2ZNatWoxbNgw2rZtS9u2bcMOS1LYAROFu38Y/XEzcA2AmZ0QZFCZKr9xh7jHV2mq1oQUm91F/Pr168f27dtVAlwSFjdRmFkjoBrwrrt/Z2a1iJTyOA9QsiiEvK2IRAamlSSkuKxYsYIbbriB6dOn07RpU8aNG8dvfvObsMOSNFHgYLaZ/QWYAHQGXjWzQcBbwKeA/g8rhP32jNDAtCTZpk2b+OSTT3j44Yd5++23lSSkUOK1KC4Est39JzM7Evg6+nxJoic3s3bA/UApYJy7j8jnmBbASKAM8J27Ny9E/GlB5cAlDIsWLWLKlCncfvvte4r4VaxYMeywJA3Fmx673d1/AnD3DcBnhUwSpYBRRNZeZAFXmllWnmMqAw8DHd29FnBZIeNPebHblCpJSDL88ssv3H333dSvX5977713TxE/JQkpqngtilPNbHcpcQNOjnmOux9oRc6ZwDJ3Xw5gZs8QaaUsijnmKmCSu6+MnnNdIeNPefu0JkKORTLf7Nmzuf7665k3bx6dOnXi/vvvVxE/OWjxEsUleZ4/VMhzVwNWxTxfTWTv7Vi/AcqY2dvAYcD97v5k3hOZWTegG0D16tULGUZqUGtCgrZ161batm1L+fLlmTx5Mh07dgw7JMkQ8YoCvnGQ57b8TpvP9zcEWgGHAu+b2Qfuvk8tKXcfC4wFyMnJSZv9ulXIT5Lhk08+oV69elSsWJHnn3+eunXrUrly5bDDkgySSAmPoloNnBjz/AQiA+J5j3nV3be6+3fAO0B2gDElzX4znUSK2Y8//kiPHj1o2LAh48ePB6BZs2ZKElLsgkwUHwOnm9kpZlYW6ARMyXPMZKCpmZU2swpEuqYWBxhT0mimkwRp6tSp1KpVizFjxtCvXz8uuSRvT7FI8Uk4UZhZoYrAuPtOoBfwGpGb/3/cfaGZdY/WjMLdFwOvAvOAj4hMoV1QmO9JRZrpJEEaMGAA559/PocffjizZs3iH//4h2Y0SaAOWMLDzM4E/kWkxlN1M8sGurp77wN91t2nAlPzvPZInud/B/5emKBTnWY6SXFzd3JzcylVqhStWrWifPny3HHHHSriJ0mRSIviAeAC4HsAd/8UlRkvkFoTUtzWrFnD73//e4YMGQJAmzZtuOuuu5QkJGkSSRSHuPtXeV7bFUQw6Uw70Ulxc3ceffRRsrKymDZtGkcffXTYIUkJlUiZ8VXR7iePrrbuDWgr1DwKW/BPJJ4vv/yS66+/nrfeeosWLVrw6KOPctppp4UdlpRQiSSKG4l0P1UHvgWmR1+T/GijISkGW7ZsYd68eYwZM4auXbtyyCFBTlAUiS+RRLHT3TsFHolICbdgwQKmTJnCHXfcQZ06dVi5ciUVKlQIOyyRhMYoPjazqWb2BzM7LPCIREqYX375hbvuuosGDRrwz3/+c08RPyUJSRUHTBTu/mvgbiKlNuab2QtmphZGjNiZTiKF8fHHH9OwYUOGDh3KZZddxqJFi1TET1JOQh2f7j7L3fsADYAfiWxoJFGa6SRFsXXrVtq1a8fGjRuZMmUKEyZMoEqVKmGHJbKfAyYKM6tkZp3N7EUiq6fXA40DjywNaaaTJGL27Nnk5uZSsWJFJk+ezMKFC+nQoUPYYYkUKJHB7AXAi8Df3H3mgQ4uCfLufy2SiE2bNnHbbbcxduxYnnjiCbp06UKTJk3CDkvkgBJJFKe6e27gkaSR/JKEynVIPC+++CLdu3fnm2++oX///lx66aVhhySSsAIThZn9w91vAf5rZvvtAZHADneZb+imPT8qSUhBbr31Vu69917q1KnDCy+8QKNGjcIOSaRQ4rUono3+t7A724mUeO7Orl27KF26NG3atOHwww9nwIABlC1bNuzQRAot3g53H0V/rOnu+yQLM+sFHOwOeGlHYxOSiNWrV3PjjTdSt25dhg8fTuvWrWndunXYYYkUWSLTY6/L57XrizuQdJC3npNIrNzcXMaMGUNWVhZvvvkmxx57bNghiRSLeGMUVxDZle4UM5sU89ZhwA9BB5bSVM9J8li+fDnXXXcdM2bMoFWrVowdO5ZTTz017LBEikW8MYqPiOxBcQIwKub1zcCcIINKFepqkkRt3bqVRYsWMW7cOK677jrMLOyQRIpNvDGKL4EviVSLLZE0DVbimT9/PpMnT+bOO++kTp06fPXVVxx66KFhhyVS7OJ1Pc1w9+ZmthGInR5rgLv7kYFHlyo0DVZi/PzzzwwfPpy//OUvHHHEEXTr1o2qVasqSUjGijeYvXu706OBKjGP3c9FSpwPPviABg0aMGzYMK688koWL16sIn6S8eJ1Pe1ejX0i8LW7/2JmTYC6wHgixQEzjsYlpCBbt27l/PPPp2LFikydOpX27duHHZJIUiQyPfYFItug/hp4EqgJ/DvQqEKSX5LQNFj58MMP9xTxe/HFF1m4cKGShJQoiSSKXHffAVwMjHT33mRoV/0+5cKHbopMg1VF2BLrhx9+oGvXrpx99tmMHz8egMaNG3PYYdq/S0qWhLZCNbPLgGuA30dfKxNcSOFTcpAXXniBHj16sG7dOgYMGMBll10WdkgioUl0ZXZLImXGl5vZKcDTwYaVfNqlTnbr168fF110EVWrVuXDDz9kxIgRmtEkJdoBWxTuvsDM+gCnmdkZwDJ3Hx58aMkV2+2Ukf1qEldsEb/f/e53HHXUUdx2222UKZPRjWeRhJj7fhXE9z3ArCnwFLCGyBqKY4Fr3P294MPbX05Ojs+ePbvYzrffAHbMmgkpGVauXEn37t2pX78+w4dn3N9AIgCY2f/cPacon02k6+mfwO/c/Vx3bwycD9xflC9LNXmThGY4lSy5ubk8/PDD1KpVixkzZnD88ceHHZJISkpkMLusuy/a/cTdF5tZRhTV36e7qedL6nIqQZYtW8Z1113HzJkzad26NWPHjuXkk08OOyyRlJRIovjEzMYQ6X4C6EyGFQXULKeSZ/v27SxdupT/+7//4w9/+IOK+InEkUii6A70AW4jMkbxDvBgkEEFTauvS6a5c+cyefJkhgwZQu3atVmxYgXly5cPOyyRlBd3jMLM6gDtgOfdvaO7d3D3v7v79uSEFwyNS5Qs27dvZ9CgQeTk5DB69GjWrVsHoCQhkqACE4WZ3UGkfEdn4HUzy2+nu/SmldcZb9asWdSvX5977rmHq6++mkWLFqmIn0ghxet66gzUdfetZlYFmAo8lpywRA7e1q1b6dChA5UqVeLVV1+lbdu2YYckkpbiJYqf3X0rgLuvN7NEptKKhO7999/nrLPOomLFirz00kvUrl1b9ZlEDkK8m/+pZjYp+nge+HXM80lxPreHmbUzsyVmtszMbo9zXCMz22Vmlxb2FxDZbePGjVx33XU0btyYp56KTNI755xzlCREDlK8FsUleZ4/VJgTm1kpIntttwZWAx+b2ZTYNRkxx/0VeK0w5xeJNWnSJHr27Mn69esZOHAgV1xxRdghiWSMeBsXvXGQ5z6TSF2o5QBm9gxwIbAoz3G9gf8CjQ7y+6SE6tu3LyNHjqRevXpMnTqV+vXrhx2SSEZJZB1FUVUDVsU8Xw2cFXuAmVUDLgLOI06iMLNuQDeA6tWrF3ugkn5ii/hdcMEFVK1alf79+6uIn0gAghygzm+pa94KhCOBAe6+K96J3H2su+e4e06VKkXfrnvNqAtg6K+K/HlJDStWrKBdu3YMHjwYgFatWjFw4EAlCZGAJJwozKxcIc+9msh+27udAHyd55gc4BkzWwFcCjxsZr8nIFpol95yc3N58MEHqV27NrNmzeKkk04KOySREuGAXU9mdibwL+BXQHUzywa6RrdEjedj4PToRkdrgE7AVbEHuPspMd/zOPCSu79QqN+gKIZuUgHANPP5559z7bXX8t5779GuXTseeeQRJQqRJEmkRfEAcAHwPYC7f0pkx7u43H0n0IvIbKbFwH/cfaGZdTez7kUPWUqiX375hS+++IInn3ySqVOnKkmIJFEig9mHuPtXeaprxh1T2M3dpxJZ0R372iMFHPvHRM4pJcecOXOYPHkyQ4cOpVatWqxYsYJy5QrbAyoiByuRFsWqaPeTm1kpM7sZWBpwXFKCbd++nYEDB9KoUSPGjBnD+vXrAZQkREKSSKK4EegHVAe+Bc6OviZS7N59912ys7MZMWIEXbp0YdGiRRzMTDcROXgH7Hpy93VEBqJFArVlyxYuvPBCDj/8cKZNm0br1q3DDklESGzW06Psv/4Bd+8WSERS4rz77rs0btyYSpUq8fLLL1O7dm0qVaoUdlgiEpVI19N04I3o4z2gKvBzkEFJyfD999/TpUsXmjZtuqeI39lnn60kIZJiEul6ejb2uZk9BbweWESS8dydiRMn0qtXLzZs2MDgwYPp1Em9myKpqii1nk4B0moSu/bITi19+/bl/vvvp2HDhkybNo3s7OywQxKROBIZo9jI3jGKQ4ANQIF7S6SivKU7tCo7+dydnTt3UqZMGTp27Mjxxx9Pv379KF06yLqUIlIc4v4rtcgqu2wiJTgAct19v4HtVLZm1AV7E4NKd4Tiyy+/pFu3bjRs2JARI0Zw3nnncd5554UdlogkKO5gdjQpPO/uu6KPtEoSsLc1oSKAybdr1y7uv/9+ateuzYcffsipp54adkgiUgSJtPs/MrMG7v5J4NEEqFrPl8IOoURZunQpf/zjH3n//fdp3749Y8aM4cQTTzzwB0Uk5RSYKMysdLSwXxPgBjP7AthKZJ8Jd/cGSYpR0tDOnTv56quvGD9+PFdddRV5aoWJSBqJ16L4CGgABLY/hGSW2bNnM3nyZIYNG0ZWVhbLly9XfSaRDBBvjMIA3P2L/B5Jik/SwE8//cRtt93GWWedxWOPPaYifiIZJl6LooqZ9SvoTXe/L4B4JM3MmDGDrl27smzZMm644Qb+9re/Ubly5bDDEpFiFC9RlAIqkf/e1yJs2bKFiy++mMqVK/PGG29oyqtIhoqXKNa6+5+TFkkA9llDIcVm5syZnHvuuVSqVIlXXnmFWrVqUbFixbDDEpGAHHCMIp1pDUXx+u6777j66qtp1qzZniJ+Z555ppKESIaL16JolbQoAqY1FAfH3fnPf/5D79692bhxI0OGDFERP5ESpMBE4e4bkhmIpK6bbrqJBx98kEaNGvHGG29Qp06dsEMSkSRSRTbJl7uzY8cOypYty0UXXcRJJ53EzTffTKlSpcIOTUSSLJGNi6SE+eKLL2jVqhV33nknAC1btuSWW25RkhApoTI2UawZdUHYIaSdXbt2cd9991GnTh3+97//UaNGjbBDEpEUkLFdT7EznjRF9sA+++wz/vCHP/DRRx/RoUMHRo8eTbVqunIikqGJInb9hGY8JSY3N5evv/6ap59+miuuuEJF/ERkj4zsetL6icR89NFHDBo0CICsrCy++OILOnXqpCQhIvvIuEQROzah1kT+tm3bRv/+/TnnnHN44okn9hTxK1u2bMiRiUgqyrhEodZEfG+99RZ16tThH//4BzfccAMLFy6kSpUqYYclIiksI8coQK2J/GzZsoXLLruMypUr89Zbb9GiRYuwQxKRNJBRLQpNic3f22+/TW5u7p4ifvPmzVOSEJGEZVSiULfTvtavX8+VV15Jy5YtGT9+PACNGjWiQoUKIUcmIukkI7ueSnq3k7vz9NNP06dPHzZv3sywYcNUxE9EiiwjE0VJ17t3b0aNGsXZZ5/Nv/71L7KyssIOSUTSmBJFhsjNzWXnzp2ULVuWSy+9lNNOO43evXurPpOIHLRAxyjMrJ2ZLTGzZWZ2ez7vdzazedHHLDPLDjKeTPX5559z3nnn7Vk816JFC1V6FZFiE1iiMLNSwCigPZAFXGlmeftAvgSau3tdYBgwtqjfVxJnPO3cuZN7772XunXrMnfuXGrWrBl2SCKSgYLsejoTWGFdfGwAAA20SURBVObuywHM7BngQmDR7gPcfVbM8R8AJxT1y0paEcDFixfTpUsXZs+ezYUXXsjDDz/M8ccfH3ZYIpKBgkwU1YBVMc9XA2fFOf564JX83jCzbkA3gOrVq8f/0hI04+nbb7/l2Wef5bLLLlN9JhEJTJBjFPnduTzfA81aEkkUA/J7393HunuOu+eU5HITH3zwAQMHDgSgZs2afPHFF1x++eVKEiISqCATxWrgxJjnJwBf5z3IzOoC44AL3f37AONJW1u3bqVv3740btyYCRMm7CniV6ZMmZAjE5GSIMhE8TFwupmdYmZlgU7AlNgDzKw6MAm4xt2XBhhL2po+fTq1a9dm5MiR9OjRQ0X8RCTpAhujcPedZtYLeA0oBTzm7gvNrHv0/UeA/wccBTwc7T7Z6e45QcWUbrZs2UKnTp048sgjeeedd2jaVKVJRCT5Al1w5+5Tgal5Xnsk5ueuQNcgY0hHb775Js2bN6dSpUq89tprZGVlceihh4YdloiUUBlRFDBT1lB8++23XH755bRq1WpPEb+GDRsqSYhIqDIiUaR71Vh356mnniIrK4vJkyczfPhwrrrqqrDDEhEBMqzWU7quoejZsyejR4/mnHPO4V//+pdWWItISsmoRJFOcnNz2bFjB+XKleOKK66gZs2a9OjRQ/WZRCTlZETXU7pZsmQJzZs331PEr3nz5qr0KiIpS4kiiXbs2MGIESPIzs5mwYIF1KlTJ+yQREQOSF1PSbJw4UKuueYa5syZw8UXX8yoUaM49thjww5LROSAlCiSpFSpUmzYsIGJEydyySWXhB2OiEjC0r7rKZXXUMyaNYsBAyJ1Ds844wyWLVumJCEiaSftE0UqrqHYsmULffr0oUmTJjz77LN89913AJQurQaciKSftE8Uu6XKGopp06ZRu3ZtHnroIXr16sWCBQs4+uijww5LRKTI9CduMdqyZQudO3fmqKOOYubMmZx77rlhhyQictDSukWRKuMTr7/+Ort27aJSpUpMmzaNuXPnKkmISMZI60QR9vjE2rVrueSSS2jTpg0TJkwAoH79+pQvXz6UeEREgpDWiWK3ZI9PuDuPP/44WVlZvPzyy4wYMUJF/EQkY2mMoghuvPFGxowZQ5MmTRg3bhw1atQIOyQRkcAoUSQotojfVVddRd26denevTuHHJIRjTIRkQKl7V0umQPZixcvpmnTptxxxx0ANGvWjB49eihJiEiJkLZ3umQMZO/YsYN77rmHevXq8dlnn1G/fv3AvktEJFWlfddTUAPZCxcu5Oqrr2bu3LlcdtllPPjggxxzzDGBfJeISCpL+0QRlNKlS7Np0yYmTZrERRddFHY4IiKhSduupyDMnDmT/v37A1CjRg2WLl2qJCEiJZ4SBbB582Z69uxJs2bNmDRpkor4iYjEKPGJ4pVXXqFWrVqMHj2am2++mfnz56uIn4hIjLT8k3nNqAuoVgzn2bx5M126dKFq1arMmjWLs88+uxjOKiKSWdKyRXEwU2PdnVdffZVdu3Zx2GGHMX36dD755BMlCRGRAqRlotitsFNj165dy8UXX0z79u33FPHLzs6mXLlyQYQnIpIR0jpRJMrdeeyxx6hZsyavvvoqf/vb31TET0QkQWk5RlFY3bt3Z+zYsTRr1oxx48Zx+umnhx2SiEjayNhEsWvXLnbs2EH58uW5+uqrqV+/Pt26dVN9JhGRQsrIu+bChQs599xz9xTxa9q0qSq9iogUUUbdOX/55ReGDRtG/fr1WbZsGY0aNQo7JBGRtJcxXU/z58+nc+fOzJ8/n06dOvHAAw9QpUqVsMMSEUl7GZMoypYty7Zt25g8eTIdO3YMOxwRkYyRdl1PO9Yt2/PzjBkzuOWWW4BIEb8lS5YoSYiIFLNAE4WZtTOzJWa2zMxuz+d9M7MHou/PM7MGBzpnmZ2b+fFnp8sblWnRogUvvPDCniJ+pUqVCuC3EBEp2QJLFGZWChgFtAeygCvNLCvPYe2B06OPbsDoA513089OrYe3MGHWavr166cifiIiAQtyjOJMYJm7Lwcws2eAC4FFMcdcCDzp7g58YGaVzew4d19b0ElX/OD8+phKzJo2nbPOOivA8EVEBIJNFNWAVTHPVwN57+z5HVMN2CdRmFk3Ii0OgJ+XrN2yQEX8ADga+C7sIFKErsVeuhZ76VrsVaOoHwwyUVg+r3kRjsHdxwJjAcxstrvnHHx46U/XYi9di710LfbStdjLzGYX9bNBDmavBk6MeX4C8HURjhERkRAFmSg+Bk43s1PMrCzQCZiS55gpQJfo7KezgU3xxidERCT5Aut6cvedZtYLeA0oBTzm7gvNrHv0/UeAqcDvgGXANuDaBE49NqCQ05GuxV66FnvpWuyla7FXka+FRSYciYiI5C/tVmaLiEhyKVGIiEhcKZsogij/ka4SuBado9dgnpnNMrPsMOJMhgNdi5jjGpnZLjO7NJnxJVMi18LMWpjZXDNbaGYzkh1jsiTwb+RXZvaimX0avRaJjIemHTN7zMzWmdmCAt4v2n3T3VPuQWTw+wvgVKAs8CmQleeY3wGvEFmLcTbwYdhxh3gtGgNHRH9uX5KvRcxxbxKZLHFp2HGH+P9FZSKVEKpHn1cNO+4Qr8UdwF+jP1cBNgBlw449gGvRDGgALCjg/SLdN1O1RbGn/Ie7/wLsLv8Ra0/5D3f/AKhsZsclO9AkOOC1cPdZ7r4x+vQDIutRMlEi/18A9Ab+C6xLZnBJlsi1uAqY5O4rAdw9U69HItfCgcPMzIBKRBLFzuSGGTx3f4fI71aQIt03UzVRFFTao7DHZILC/p7XE/mLIRMd8FqYWTXgIuCRJMYVhkT+v/gNcISZvW1m/zOzLkmLLrkSuRYPATWJLOidD9zk7rnJCS+lFOm+maobFxVb+Y8MkPDvaWYtiSSKJoFGFJ5ErsVIYIC774r88ZixErkWpYGGQCvgUOB9M/vA3ZcGHVySJXIt2gJzgfOAXwOvm9lMd/8x6OBSTJHum6maKFT+Y6+Efk8zqwuMA9q7+/dJii3ZErkWOcAz0SRxNPA7M9vp7i8kJ8SkSfTfyHfuvhXYambvANlApiWKRK7FtcAIj3TULzOzL4EzgI+SE2LKKNJ9M1W7nlT+Y68DXgszqw5MAq7JwL8WYx3wWrj7Ke5+srufDEwEemRgkoDE/o1MBpqaWWkzq0CkevPiJMeZDIlci5VEWlaY2TFEKqkuT2qUqaFI982UbFF4cOU/0k6C1+L/AUcBD0f/kt7pGVgxM8FrUSIkci3cfbGZvQrMA3KBce6e77TJdJbg/xfDgMfNbD6R7pcB7p5x5cfN7GmgBXC0ma0GhgBl4ODumyrhISIicaVq15OIiKQIJQoREYlLiUJEROJSohARkbiUKEREJC4lCkk50aqvc2MeJ8c59uSCKmUW8jvfjlYf/dTM3jOzGkU4R/fdZTLM7I9mdnzMe+PMLKuY4/zYzOol8Jmbo+soRIpEiUJS0U/uXi/msSJJ39vZ3bOBJ4C/F/bD0bULT0af/hE4Pua9ru6+qFii3BvnwyQW582AEoUUmRKFpIVoy2GmmX0SfTTO55haZvZRtBUyz8xOj75+dczrY8ys1AG+7h3gtOhnW5nZHDObH631Xy76+ggzWxT9nnujrw01s/4W2QMjB5gQ/c5Doy2BHDO70cz+FhPzH83swSLG+T4xBd3MbLSZzbbIfgt3RV/rQyRhvWVmb0Vfa2Nm70ev43NmVukA3yMlnBKFpKJDY7qdno++tg5o7e4NgCuAB/L5XHfgfnevR+RGvdrMakaPPzf6+i6g8wG+vwMw38zKA48DV7h7HSKVDG40syOJVKit5e51gbtjP+zuE4HZRP7yr+fuP8W8PRG4OOb5FcCzRYyzHRBbnmRQdEV+XaC5mdV19weI1PJp6e4tzexo4E7gt9FrORvod4DvkRIuJUt4SIn3U/RmGasM8FC0T34XkRLaeb0PDDKzE4jsw/C5mbUiUkH142h5k0MpeJ+KCWb2E7CCyJ4WNYAvY+pnPQH0JFKyejswzsxeBl5K9Bdz9/VmtjxaZ+fz6He8Fz1vYeKsSKRcRewOZZebWTci/66PA7KIlO+IdXb09fei31OWyHUTKZAShaSLvsC3RKqfHkLkRr0Pd/+3mX0InA+8ZmZdidT1ecLdBybwHZ3dffbuJ2Z2VH4HRWsLnUmkyFwnoBeR8tWJeha4HPgMeN7d3SJ37YTjJLKL2whgFHCxmZ0C9AcauftGM3scKJ/PZw143d2vLES8UsKp60nSxa+AtdHNZq4h8tf0PszsVGB5tLtlCpEumDeAS82savSYI83spAS/8zPgZDM7Lfr8GmBGtE//V+4+lchAcX4zjzYDhxVw3knA74EriSQNChunu+8g0oV0drTb6nBgK7DJItVR2xcQywfAubt/JzOrYGb5tc5E9lCikHTxMPAHM/uASLfT1nyOuQJYYGZziew18GR0ptGdwDQzmwe8TqRb5oDcfTuR6prPRauO5hLZOe8w4KXo+WYQae3k9TjwyO7B7Dzn3UhkL+uT3P2j6GuFjjM69vEPoL+7fwrMARYCjxHpztptLPCKmb3l7uuJzMh6Ovo9HxC5ViIFUvVYERGJSy0KERGJS4lCRETiUqIQEZG4lChERCQuJQoREYlLiUJEROJSohARkbj+P03q/i9gw36QAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "y_scores2 = cross_val_predict(lr_penalty,X_train,y_train,cv=5,method=\"decision_function\")\n",
    "\n",
    "\n",
    "from sklearn.metrics import roc_curve,auc\n",
    "\n",
    "# y_scores = cross_val_p\n",
    "fpr,tpr,thresholds = roc_curve(y_train,y_scores2)\n",
    "\n",
    "def plot_roc_curve(lables,predict_prob):\n",
    "    plt.plot(fpr,tpr,label = None)\n",
    "    plt.plot(fpr,tpr,linewidth = 2,label = \"label\")\n",
    "    plt.plot([0,1],[0,1],'k--')\n",
    "    plt.axis([0,1,0,1])\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "plot_roc_curve(fpr,tpr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.69314718, -0.62766473],\n",
       "       [-0.63596759, -0.51494876],\n",
       "       [-0.47851451, -0.47676173],\n",
       "       [-0.47596795, -0.47615971],\n",
       "       [-0.47640619, -0.47643827],\n",
       "       [-0.47646752, -0.47647175],\n",
       "       [-0.47647252, -0.47647516]])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(test_means).reshape(n_Cs,number_penaltys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.69314718, -0.62766473, -0.63596759, -0.51494876, -0.47851451,\n",
       "       -0.47676173, -0.47596795, -0.47615971, -0.47640619, -0.47643827,\n",
       "       -0.47646752, -0.47647175, -0.47647252, -0.47647516])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
